package util.content;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.PreparedStatement;
import java.sql.Statement;
import java.util.Iterator;
import java.util.List;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import replaceWords.WeiYuanChuangXML;

public class ContentSpider {

	public static void main(String[] args) {
		/*//抓取的是本地的数据库不需要延时了设成1毫秒
		final int contentSpiderInterval = 1;*/
		final int contentSpiderInterval = 3000;
		final long errorInterval = 10000;
		String titlePub = null;
		String contentPub = null;
		Connection conn = null;
		Statement stmt = null;
		PreparedStatement pstmt = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		String dBName = null;
		String dBHost = null;
		String wordHost = null;
		String wordDBName = null;
		String contentString = null;
		String sourceNet = null;
		int repeatTimesContent = 0;
		// 把程序包在一个大的循环里出错后重新进入一个新循环，继续运行不中断
		while (true) {
			// 初始化伪原创对象
			WeiYuanChuangXML wyc = new WeiYuanChuangXML();
			// 初始化内容处理器
			ContentRex cr = new ContentRex();
			try {
				Class.forName("com.mysql.jdbc.Driver");
				SAXBuilder builder = new SAXBuilder(false);
				List sourceSiteParameterList = null;
				Document doc = builder.build("spiderParameter.xml");
				Element parameter = doc.getRootElement();
				sourceSiteParameterList = parameter.getChildren("sourceSite");
				for (Iterator iter = sourceSiteParameterList.iterator(); iter.hasNext();) {
					Element sourceSiteParameter = (Element) iter.next();
					// 拿到数据库和抓取需要的各项参数
					dBName = sourceSiteParameter.getChildTextTrim("dBName");
					dBHost = sourceSiteParameter.getChildTextTrim("host");
					wordHost = sourceSiteParameter.getChildTextTrim("wordHost");
					wordDBName = sourceSiteParameter.getChildTextTrim("wordDBName");
					sourceNet = sourceSiteParameter.getChildTextTrim("sourceNet");
					repeatTimesContent = Integer.parseInt(sourceSiteParameter.getChildTextTrim("repeatTimesContent"));
					wyc.setWordDBName(wordDBName);
					wyc.setWordHost(wordHost);

					// 抓取需要的参数哪去完毕后，查询tagurl表拿到需要抓取的网页链接插入到url表
					conn = DriverManager.getConnection("jdbc:mysql://" + dBHost + "/" + dBName + "?user=root");
					connGetSite = conn;
					stmt = conn.createStatement();
					stmtGetSite = connGetSite.createStatement();
					// 从url表中查询需要采集的链接
					rsGetSite = stmtGetSite.executeQuery("select id,url from url where urlno = '" + sourceNet + "' and sw = 0");
					while (rsGetSite.next()) {
						// 这里有放置一个计时器
						Long btime = System.currentTimeMillis();
						// 初始化抓取器类
						SpiderContentTxt sct = new SpiderContentTxt();
						//SpiderContentTxtMend sct = new SpiderContentTxtMend();
						/*修复版本的代码
						*sct.setdBName(dBName);*/
						sct.setRepeatTimesContent(repeatTimesContent);
						sct.SpiderPageContent(rsGetSite.getString("url"));
						contentString = sct.getWebSiteBodyString();
						String title = sct.getTitle();
						sct.clear();
						sct = null;
						// 抓取对象出错的话执行下面一段，copy=-3
						if (contentString.equalsIgnoreCase("0")) {
							pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,copy) values (?,?,?,?,?,?)");
							pstmt.setInt(1, 0);
							pstmt.setString(2, title);
							pstmt.setString(3, contentString);
							pstmt.setString(4, sourceNet);
							pstmt.setString(5, rsGetSite.getString("url"));
							// 抓取出错的设置copy=-3
							pstmt.setInt(6, -3);
							pstmt.execute();
						}
						// 抓取正常的情况下，对抓取下的字符串进行处理
						else {
							// contentString、title处理，拿出有效的content和title
							// contnetPub和title是有效的发布出去的最终内容，其中content还经过伪原创处理
							// 调用ContentRex提取内容
							cr.setBodyString(contentString);
							cr.setTitle(title);
							titlePub = cr.getTitle();
							contentPub = cr.getContentString();
							// System.out.println(contentPub);
							// 如果title长度小于5或content长度小于200文章也不发布，方法是设置copy = -2
							if (titlePub.length() < 5 || contentPub.length() < 200) {
								// 检验一下contentPub是不是9527
								if (contentPub.equalsIgnoreCase("9527")) {
									pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,copy) values (?,?,?,?,?,?)");
									pstmt.setInt(1, 0);
									pstmt.setString(2, title);
									pstmt.setString(3, contentString);
									pstmt.setString(4, sourceNet);
									pstmt.setString(5, rsGetSite.getString("url"));
									// copy 设置为-1表示文章还在征答中过几天在抓取
									pstmt.setInt(6, -1);
									pstmt.execute();

								} else {
									pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,titlePub,contentPub,copy) values (?,?,?,?,?,?,?,?)");
									pstmt.setInt(1, 0);
									pstmt.setString(2, title);
									pstmt.setString(3, contentString);
									pstmt.setString(4, sourceNet);
									pstmt.setString(5, rsGetSite.getString("url"));
									pstmt.setString(6, titlePub);
									pstmt.setString(7, contentPub);
									// copy 设置为-2表示文章内容太少或采集不到
									pstmt.setInt(8, -2);
									pstmt.execute();
								}
							} else {
								// 调用伪原创
								wyc.setContent(contentPub);
								wyc.wyc();
								contentPub = wyc.getWordOK();
								// System.out.println(contentPub);
								pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,titlePub,contentPub) values (?,?,?,?,?,?,?)");
								pstmt.setInt(1, 0);
								pstmt.setString(2, title);
								pstmt.setString(3, contentString);
								pstmt.setString(4, sourceNet);
								pstmt.setString(5, rsGetSite.getString("url"));
								pstmt.setString(6, titlePub);
								pstmt.setString(7, contentPub);
								pstmt.execute();
							}
						}
						// 更新sw位
						stmt.executeUpdate("update url set sw = 1 where id = " + rsGetSite.getInt(1));
						System.out.println("************" + dBName + "现在抓取到url表的第" + (rsGetSite.getInt(1) + 1) + "个数据");
						Long etime = System.currentTimeMillis();
						// 如果不到2.5秒就加一段延时
						if ((etime - btime) < contentSpiderInterval) {
							Thread.sleep(contentSpiderInterval - (etime - btime));
						}
					}
				}
				// 因为程序包在一个大的死循环里，完成后只能用这个break跳出循环
				System.out.println("---------抓取Content完成，下一步break出循环----------");
				break;
			}

			// 对所有catch的处理方式都是简单打印exception名字，暂停10秒进入下一次循环，也就是重新启动程序
			catch (JDOMException e) {
				System.out.println("-----JDOMException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (OutOfMemoryError e) {
				// 捕获OutOfMemoryError后所有引用的对象都置为null,暂停30秒(其他错误只停10秒)
				System.out.println("-----OutOfMemoryError:" + e.getMessage());
				try {
					wyc = null;
					cr = null;
					conn = null;
					stmt = null;
					pstmt = null;
					connGetSite = null;
					stmtGetSite = null;
					rs = null;
					rsGetSite = null;
					Thread.sleep(errorInterval * 3);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (IOException e) {
				System.out.println("-----IOException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (ClassNotFoundException e) {
				System.out.println("-----ClassNotFoundException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (SQLException e) {
				System.out.println("-----SQLError:" + e.getMessage());
				/*
				 * try { Thread.sleep(errorInterval); continue; } catch
				 * (InterruptedException e1) { // TODO Auto-generated catch
				 * block e1.printStackTrace(); }
				 */
				e.printStackTrace();
			} catch (InterruptedException e) {
				System.out.println("-----InterruptedException:" + e.getMessage());
				try {
					Thread.sleep(errorInterval);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				// e.printStackTrace();
			} catch (Exception e) {
				/*
				 * System.out.println("ContentSpider-----Exception:" +
				 * e.getMessage()); try { Thread.sleep(errorInterval); continue;
				 * } catch (InterruptedException e1) { // TODO Auto-generated
				 * catch block e1.printStackTrace(); }
				 */
				e.printStackTrace();
			} finally {
				try {
					if (rs != null) {
						rs.close();
						rs = null;
					}
					if (stmt != null) {
						stmt.close();
						stmt = null;
					}
					if (conn != null) {
						conn.close();
						conn = null;
					}
					if (rsGetSite != null) {
						rsGetSite.close();
						rsGetSite = null;
					}
					if (stmtGetSite != null) {
						stmtGetSite.close();
						stmtGetSite = null;
					}
					if (connGetSite != null) {
						connGetSite.close();
						connGetSite = null;
					}
					if (pstmt != null) {
						pstmt.close();
						pstmt = null;
					}
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

}
